package com.jackingod.crawler;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.MalformedURLException;
import java.net.URL;

public class NewsCrawler {

	private String url;
	public String htmlContent;
	public String NewsTitle;
	public String NewsContent;

	private String NewLine = System.getProperty("line.separator");

	public void start(String url) {
		init();
		this.url = url;
		this.startCrawling();
		this.extractNews();
	}
	
	private void init() {
		this.htmlContent = null;
		this.NewsTitle = null;
		this.NewsContent = null;
	}

	private void startCrawling() {
		StringBuffer sb = new StringBuffer();
		InputStreamReader isr = null;
		BufferedReader br = null;

		try {
			URL newsUrl = new URL(url);

			isr = new InputStreamReader(newsUrl.openStream());
			while (!isr.ready()) {
				// wait until the full web page has been downloaded.
			}

			br = new BufferedReader(isr);
			while (br.ready()) {
				sb.append(br.readLine()).append(NewLine);
			}

			this.htmlContent = sb.toString();

		} catch (MalformedURLException e) {
			e.printStackTrace();
		} catch (IOException e) {
			e.printStackTrace();
		} finally {
			try {
				if (br != null)
					br.close();
				if (isr != null)
					isr.close();
			} catch (IOException e) {
				e.printStackTrace();
			}
		}
	}

	private void extractNews() {
		if (this.htmlContent == null)
			return;

		try {
			// find the title tag
			int titleStartIdx = this.htmlContent.indexOf("<!-- begin_t -->");
			int titleEndIdx = this.htmlContent.indexOf("<!-- end_t -->");
			this.NewsTitle = this.htmlContent.substring(titleStartIdx + 16,
					titleEndIdx);
		} catch (StringIndexOutOfBoundsException e) {
			e.printStackTrace();
		}

		try {
			// find news cotent tag
			int cxtStartIdx = this.htmlContent.indexOf("<!-- begin_ct -->") 
					+ 17;
			int cxtEndIdx = this.htmlContent.indexOf("<!-- end_ct -->");
			String context = this.htmlContent.substring(cxtStartIdx, cxtEndIdx);
			
			// remove table
			int tblStartIdx = context.indexOf("<table");
			if (tblStartIdx > -1) {
				StringBuffer sbCxt = new StringBuffer();
				sbCxt.append(context.substring(0, tblStartIdx));
				
				int tblEndIdx = context.indexOf("</table>") + 8;
				sbCxt.append(context.substring(tblEndIdx));
				context = sbCxt.toString();
			}
			
			// extract paragraph between <p> and </p>
			StringBuffer sbPara = new StringBuffer();
			int paraStartIdx = context.indexOf("<p>");
			int paraEndIdx = -1;
			while (paraStartIdx > -1) {
				paraEndIdx = context.indexOf("</p>");
				String para = context.substring(paraStartIdx + 3, 
						paraEndIdx);
				sbPara.append(para).append(NewLine);
				context = context.substring(paraEndIdx + 4);
				paraStartIdx = context.indexOf("<p>");
			}
			
			context = sbPara.toString().trim();
			
			this.NewsContent = context;
		} catch (StringIndexOutOfBoundsException e) {
			e.printStackTrace();
		}
	}
}
